# -*- coding: utf-8 -*-
import urllib, urllib2 
import re, sys, os
import datetime, time
import traceback
import urlparse
import mechanize
import HTMLParser
import binascii, hashlib
import cStringIO as StringIO
import argparse
import paramiko
import logging
import Image

from lxml       import etree
from urlparse   import urljoin
from termcolor  import cprint
from mongolog.handlers import MongoHandler
#log = logging.getLogger('development')
#log.setLevel(logging.DEBUG)
#log.addHandler(MongoHandler.to(db='log_truyentranh', collection='log'))
stringify = etree.XPath("string()")
os.umask(0000)

def getMongoLog(host, port, name):
    log = logging.getLogger(name)
    log.setLevel(logging.WARNING)
    log.addHandler(MongoHandler.to('mongolog', 'log', host=host, port=port))
    return log

def getHTML(url, output=False):
    try:
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        response    =   urllib.urlopen(url)
        html        =   response.read()
        if output: print html
        return html
    except:
        traceback.print_exc()

def getXMLTree(url, isXML=False, userAgent=False, outputHTML=False, returnHTML=False, charset=None, removeTags=None):
    try:
        ''' Hàm xây dựng cây:
            - Đọc HTML từ URL đưa vào
            - options : encoding='utf-8' override phương thức encoding
            = isXML : 
                +) False : HTMLParser
                +) True  : XMLParser
        '''
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        if userAgent:
            user_agent = ''
            if userAgent==True:
                user_agent  = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16'
            else:
                user_agent  = userAgent
            opener      = mechanize.build_opener(mechanize.HTTPRefererProcessor, mechanize.HTTPRefreshProcessor, mechanize.HTTPEquivProcessor)
            opener.addheaders = [("User-agent", user_agent)]
            response    = opener.open(url)
        else:
            response    = urllib.urlopen(url)
        '''
        if encode==None:
            html    = response.read()
        else:
            parser  = HTMLParser.HTMLParser()
            html    = response.read().decode(encode)
            html    = parser.unescape(html)
        '''
        html    = response.read()
        if removeTags!=None:
            for tag in removeTags: html = html.replace(tag, '')
        if outputHTML: print(html)
        if isXML==False: parser  = etree.HTMLParser(encoding='utf-8')
        else: parser = etree.XMLParser(encoding='utf-8')
        if charset:
            tree    = etree.parse(StringIO.StringIO(html), etree.HTMLParser(encoding=charset))
        else:
            tree    = etree.parse(StringIO.StringIO(html), parser)
        if returnHTML:
            return tree, html
        return tree
    except:
        traceback.print_exc()

def getPathWithName(src):
    ''' Hàm lấy tên ảnh qua xử lý md5(src)
        Giá trị trả về là chuỗi mã hóa md5 và đường dẫn (ex: a/b/c/d)
    '''
    try:
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        md5str = hashlib.md5(src).hexdigest()
        path = '{0}/{1}/{2}/{3}'.format(md5str[0], md5str[1], md5str[2], md5str[3])
        return md5str, path
    except:
        traceback.print_exc()
        return None, None

def saveImageWithSCP(src, prefix, localPath, ssh=None, sftp=None):
    try:
        if src==None or src=='' or ssh==None or sftp==None: return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src)
        if img_name==None: return None, None, None, None
        img_ext      = os.path.splitext(src)[1]
        img_ext      = img_ext.lower()
        if len(img_ext)>4:  
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg' 
        preg         = re.compile('jpg|png|gif|png')
        m            = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file    = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir     = '{0}/{1}'.format(localPath, path)
        source       = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)
        file_name    = '{0}.jpg'.format(img_name)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']): 
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        stdin, stdout, stderr   = ssh.exec_command("mkdir -p -m 0777 {0}".format(path_dir))
                        remote_file = path_file
                        info = rexists(sftp, remote_file)
                        if not info:
                            flgRun = False
                            for i in range(0, 3):
                                try:
                                    sftp.put(path_file, remote_file)
                                    print 'Copy file: ', path_file, ' => ', remote_file
                                    flgRun = True; break
                                except:
                                   continue 
                            if flgRun:
                                cprint('Lưu ảnh thành công!', 'green'); break
                            else: 
                                cprint('Không upload được ảnh lên', 'red'); result = None; break
                        else: cprint('Ảnh đã tồn tại trên server!', 'red')
                        cprint('Lưu ảnh thành công!', 'green'); break
                    else: os.unlink(location); result = None
                else:
                    size = os.path.getsize(path_file)
                    stdin, stdout, stderr   = ssh.exec_command("mkdir -p -m 0777 {0}".format(path_dir))
                    remote_file = path_file
                    info = rexists(sftp, remote_file)
                    if not info:
                        flgRun = False
                        for i in range(0, 3):
                            try:
                                sftp.put(path_file, remote_file)
                                print 'Copy file: ', path_file, ' => ', remote_file
                                flgRun = True; break
                            except:
                               continue 
                        if flgRun:
                            cprint('Upload ảnh thành công!', 'green')
                        else: 
                            cprint('Không upload được ảnh lên', 'red'); result = None; break
                    else: cprint('Ảnh đã tồn tại trên server!', 'red')
                    cprint('Ảnh đã tồn tại !', 'red'); result = 1; break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None

def saveImageNotHash(src, prefix, localPath, imgName=None, nameStory=None):
    try:
        if src==None or src=='': return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_ext      = os.path.splitext(src)[1]
        img_ext      = img_ext.lower()
        if len(img_ext)>4:  
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg' 
        preg         = re.compile('jpg|png|gif|png')
        m            = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        img_name = ''
        if imgName==None: img_name   = src.split('/')[-1]
        else: img_name = '{0}{1}'.format(imgName, img_ext)
        img_name     = img_name.lower()
        path_file    = '{0}/{1}'.format(localPath, img_name)
        path_dir     = localPath
        if nameStory!=None:
            source       = '{0}/{1}/{2}/{3}'.format(prefix, nameStory, localPath.split('/')[-1], img_name) 
        else:
            source       = '{0}/{1}/{2}'.format(prefix, localPath.split('/')[-1], img_name) 
        file_name    = img_name
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0; location = ''
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        #log.error('%s: content-type not exists!' % src)
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']): 
                            os.unlink(location); result = None
                            #log.error('%s: file is not image' % src)
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        #log.error('%s: size of file not equal file response' % src)
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        cprint(file_name, 'yellow')
                        cprint('Lưu ảnh thành công!', 'green');
                        break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    if size > 0:
                      cprint('Ảnh đã tồn tại !', 'red');
                      result = 1;
                    else: os.unlink(path_file); result = None
                    break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None

def saveImageWH(src, prefix, localPath):
    ''' Hàm lưu ảnh với src, prefix, local_path
        Giá trị trả về là : 
            +) result (0 chưa tồn tại, 1: ảnh đã tồn tại)
            +) source (prefix/path/file_name) Ex: /uploads/xkcn/a/b/c/d/abcd.....jpg
            +) img_name: tên ảnh là md5(src).jpg
            +) size : Dung lượng của ảnh   
    '''
    try:
        width = 0; height = 0
        if src==None or src=='': return None, None, None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src)
        if img_name==None: return None, None, None, None, None, None
        img_ext      = os.path.splitext(src)[1]
        img_ext      = img_ext.lower()
        if len(img_ext)>4:  
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg' 
        preg         = re.compile('jpg|png|gif|png')
        m            = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file    = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir     = '{0}/{1}'.format(localPath, path)
        source       = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)

        print '########### SOURCE: ', source
        file_name    = '{0}{1}'.format(img_name, img_ext)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']): 
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    width, height = Image.open(path_file).size
                    if size > 0:
                        cprint('Lưu ảnh thành công!', 'green'); break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    width, height = Image.open(path_file).size
                    cprint('Ảnh đã tồn tại !', 'red'); result = 1; break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size, width, height
    except:
        traceback.print_exc()
        return None, None, None, None, None, None

def saveImage(src, prefix, localPath):
    ''' Hàm lưu ảnh với src, prefix, local_path
        Giá trị trả về là : 
            +) result (0 chưa tồn tại, 1: ảnh đã tồn tại)
            +) source (prefix/path/file_name) Ex: /uploads/xkcn/a/b/c/d/abcd.....jpg
            +) img_name: tên ảnh là md5(src).jpg
            +) size : Dung lượng của ảnh   
    '''
    try:
        if src==None or src=='': return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src)
        if img_name==None: return None, None, None, None
        img_ext      = os.path.splitext(src)[1]
        img_ext      = img_ext.lower()
        if len(img_ext)>4:  
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg' 
        preg         = re.compile('jpg|png|gif|png')
        m            = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file    = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir     = '{0}/{1}'.format(localPath, path)
        source       = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)
        file_name    = '{0}{1}'.format(img_name, img_ext)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']): 
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        cprint('Lưu ảnh thành công!', 'green'); break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    cprint('Ảnh đã tồn tại !', 'red'); result = 1; break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None
    
def getStringWithNode(node):
    ''' Hàm lấy nội dung của một node qua strinify
        +) Loại bỏ hết nhiều whitespace, ký tự đơn: Ex: \r \n \t
    '''
    try:
        if node==None: return
        text = stringify(node).strip()
        text = ' '.join(text.split())
        return text
    except:
        traceback.print_exc()
        
def convertToText(text):
    ''' Hàm loại bỏ nhiều whitespace thành single whitespace, remove ký tự đơn \r \n \t
    ''' 
    try:
        if text==None or text=='': return
        text = ' '.join(text.split())
        return text
    except:
        traceback.print_exc()

def getMd5(text):
    try:
        if text==None or text=='': return
        return hashlib.md5(text).hexdigest()
    except:
        traceback.print_exc()

def getDatatypeName(nametype):
    ''' Hàm trả về tên của kiểu dữ liệu
        Ex: int, list, ...
    '''
    try:
        if nametype==None or nametype=='': return 
        return type(nametype).__name__
    except:
        traceback.print_exc()

def regexString(pattern, text):
    try:
        if pattern==None or text==None or pattern=='' or text=='': return
        preg    =   re.compile(pattern)
        m       =   preg.search(text)
        if m: return m
    except:
        traceback.print_exc()
        
def getCharsetWithUrl(url):
    try:
        charset     =   'UTF-8'
        if url==None or url=='': return charset
        opener      =   urllib2.urlopen(url)
        result     =   opener.headers.getparam('charset')
        if result!=None: charset = result 
        return charset
    except:        
        traceback.print_exc()

def remove_html_tags(text):
    try:
        preg = re.compile(r'<.*?>')
        return preg.sub('', text)
    except:
        traceback.print_exc()

def remove_extra_spaces(text):
    try:
        preg = re.compile(r'\s+')
        return preg.sub('', text)
    except:
        traceback.print_exc()

def toAscii(text):
    ''' Chuyển chuỗi tiếng Việt Unicode tổ hợp về dạng Ascii không dấu '''
    if text == '': return ''
    if type(text).__name__ == 'unicode': text   = text.encode('utf-8')
    listPattern = ["á|à|ả|ạ|ã|â|ấ|ầ|ẩ|ậ|ẫ|ă|ắ|ằ|ẳ|ặ|ẵ", "Á|À|Ả|Ạ|Ã|Â|Ấ|Ầ|Ẩ|Ậ|Ẫ|Ă|Ắ|Ằ|Ẳ|Ặ|Ẵ",
                   "đ", "Đ", "í|ì|ỉ|ị|ĩ", "Í|Ì|Ỉ|Ị|Ĩ", "é|è|ẻ|ẹ|ẽ|ê|ế|ề|ể|ệ|ễ", "É|È|Ẻ|Ẹ|Ẽ|Ê|Ế|Ề|Ể|Ệ|Ễ",
                   "ó|ò|ỏ|ọ|õ|ô|ố|ồ|ổ|ộ|ỗ|ơ|ớ|ờ|ở|ợ|ỡ", "Ó|Ò|Ỏ|Ọ|Õ|Ô|Ố|Ồ|Ổ|Ộ|Ỗ|Ơ|Ớ|Ờ|Ở|Ợ|Ỡ",
                   "ú|ù|ủ|ụ|ũ|ư|ứ|ừ|ử|ự|ữ", "Ú|Ù|Ủ|Ụ|Ũ|Ư|Ứ|Ừ|Ử|Ự|Ữ", "ý|ỳ|ỷ|ỵ|ỹ", "Ý|Ỳ|Ỷ|Ỵ|Ỹ"]
    rep = ['a', 'A', 'd', 'D', 'i', 'I', 'e', 'E', 'o', 'O', 'u', 'U', 'y', 'Y']
    for idx in range(len(listPattern)):
        text = re.sub(listPattern[idx], rep[idx], text)
    return text

def getArgs():
    parser  = argparse.ArgumentParser()
    parser.add_argument('-e', '--run', '-run', '-copy', action='store', dest='copy_file', help='Copy file to local => server.')
    args    = parser.parse_args()
    return  args.copy_file

def createSSHClient(server, port, user, passwd):
    try:
        client  = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(server, port, user, passwd)

        return client
    except:
        traceback.print_exc()

def rexists(sftp, path):
    try:
        sftp.stat(path)
        return True
    except:
        return False

def forceQuit():
    try:
        print 'Finished.', datetime.datetime.now()
        pid = os.getpid(); os._exit(1); os.kill(pid, 9)
    except:
        traceback.print_exc()

unicodeDungSan = 'a á à ả ã ạ ă ắ ằ ẳ ẵ ặ â ấ ầ ẩ ẫ ậ e é è ẻ ẽ ẹ ê ế ề ể ễ ệ i í ì ỉ ĩ ị o ó ò ỏ õ ọ ô ố ồ ổ ỗ ộ ơ ớ ờ ở ỡ ợ u ú ù ủ ũ ụ ư ứ ừ ử ữ ự y ý ỳ ỷ ỹ ỵ đ A Á À Ả Ã Ạ Ă Ắ Ằ Ẳ Ẵ Ặ Â Ấ Ầ Ẩ Ẫ Ậ E É È Ẻ Ẽ Ẹ Ê Ế Ề Ể Ễ Ệ I Í Ì Ỉ Ĩ Ị O Ó Ò Ỏ Õ Ọ Ô Ố Ồ Ổ Ỗ Ộ Ơ Ớ Ờ Ở Ỡ Ợ U Ú Ù Ủ Ũ Ụ Ư Ứ Ừ Ử Ữ Ự Y Ý Ỳ Ỷ Ỹ Ỵ Đ'
unicodeToHop =   'a á à ả ã ạ ă ắ ằ ẳ ẵ ặ â ấ ầ ẩ ẫ ậ e é è ẻ ẽ ẹ ê ế ề ể ễ ệ i í ì ỉ ĩ ị o ó ò ỏ õ ọ ô ố ồ ổ ỗ ộ ơ ớ ờ ở ỡ ợ u ú ù ủ ũ ụ ư ứ ừ ử ữ ự y ý ỳ ỷ ỹ ỵ đ A Á À Ả Ã Ạ Ă Ắ Ằ Ẳ Ẵ Ặ Â Ấ Ầ Ẩ Ẫ Ậ E É È Ẻ Ẽ Ẹ Ê Ế Ề Ể Ễ Ệ I Í Ì Ỉ Ĩ Ị O Ó Ò Ỏ Õ Ọ Ô Ố Ồ Ổ Ỗ Ộ Ơ Ớ Ờ Ở Ỡ Ợ U Ú Ù Ủ Ũ Ụ Ư Ứ Ừ Ử Ữ Ự Y Ý Ỳ Ỷ Ỹ Ỵ Đ'
aDungSan = unicodeDungSan.split(' ')
aToHop = unicodeToHop.split(' ')
def toUnicodeDungSan(strInp):
    res = strInp
    if type(res).__name__=='unicode': res = res.encode('utf-8') 
    for i in range(len(aToHop)):
        res = res.replace(aToHop[i], aDungSan[i])
    return res
